package com.qingguo.Test

import java.security.MessageDigest

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode}

object cjTest {

  def main(args: Array[String]): Unit = {
    /**
      * 如何使用带有Spark Dataframe的Except功能
      *
      * 回答 (2)关注 (0)查看 (1422)
      * 我想获得两个数据帧之间的差异，但只返回具有不同字段的行。例如，我有2个数据帧如下：
      *
      */

    val conf = new SparkConf().setMaster("local").setAppName("MD5")
    val sc = new SparkContext(conf)

    conf.set("spark.sql.shuffle.partitions", "1")

    // spark  sql 上下文对象  spark sql的入口
    val spark = new SQLContext(sc)

    import spark.implicits._

    val DF1 = Seq(
      (3, "Chennai", "rahman", 9846, 45000, "SanRamon"),
      (1, "Hyderabad", "ram", 9847, 50000, "SF"),
      (11, "Hyderabad", "ram", 9847, 50000, "SF")
    ).toDF("emp_id", "emp_city", "emp_name", "emp_phone", "emp_sal", "emp_site")

    val DF2 = Seq(
      (3, "Chennai", "rahman", 9846, 45000, "SanRamon"),
      (1, "Sydney", "ram", 9847, 48000, "SF")
    ).toDF("emp_id", "emp_city", "emp_name", "emp_phone", "emp_sal", "emp_site")

    val colIdx = "emp_id"

    DF1.limit(3).select(colIdx).except(DF2.select(colIdx)).show()

  }

}
